From 0931cb90781961d2e4dfe96070bf6a31470c4a86 Mon Sep 17 00:00:00 2001 From: "kaf24@firebug.cl.cam.ac.uk" Date: Thu, 31 Mar 2005 17:02:55 +0000 Subject: [PATCH] bitkeeper revision 1.1236.1.176 (424c2d3fy4zylRn8ctgTwLuwa-5b1Q) Fix SMP TLB flushing in XenLinux 2.6. Signed-off-by: Keir Fraser --- .../arch/xen/i386/kernel/pci-dma.c | 5 ++- .../arch/xen/i386/kernel/smp.c | 17 ++++++++ .../arch/xen/i386/mm/hypervisor.c | 39 ++++++++++++++++++- .../drivers/xen/balloon/balloon.c | 3 +- .../include/asm-xen/asm-i386/pgtable.h | 3 +- .../include/asm-xen/asm-i386/tlbflush.h | 14 +------ .../include/asm-xen/hypervisor.h | 8 ++++ 7 files changed, 70 insertions(+), 19 deletions(-) diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c index 1d8f781454..002eb0abd1 100644 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) @@ -78,9 +79,9 @@ xen_contig_memory(unsigned long vstart, unsigned int order) phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = pfn+i; } - xen_tlb_flush(); + flush_tlb_all(); - balloon_unlock(flags); + balloon_unlock(flags); } #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c index 7b7763848b..bcd398354e 100644 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c @@ -203,6 +203,7 @@ inline void send_IPI_mask_sequence(cpumask_t mask, int vector) #include /* must come after the send_IPI functions above for inlining */ +#if 0 /* XEN */ /* * Smarter SMP flushing macros. * c/o Linus Torvalds. @@ -441,6 +442,22 @@ void flush_tlb_all(void) on_each_cpu(do_flush_tlb_all, NULL, 1, 1); } +#else + +irqreturn_t smp_invalidate_interrupt(int irq, void *dev_id, + struct pt_regs *regs) +{ return 0; } +void flush_tlb_current_task(void) +{ xen_tlb_flush_mask(current->mm->cpu_vm_mask); } +void flush_tlb_mm(struct mm_struct * mm) +{ xen_tlb_flush_mask(mm->cpu_vm_mask); } +void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) +{ xen_invlpg_mask(vma->vm_mm->cpu_vm_mask, va); } +void flush_tlb_all(void) +{ xen_tlb_flush_all(); } + +#endif /* XEN */ + /* * this function sends a 'reschedule' IPI to another CPU. * it goes straight through and wastes no time serializing diff --git a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c index 1ac796d9fd..82493c1236 100644 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) @@ -94,6 +95,42 @@ void xen_invlpg(unsigned long ptr) BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0); } +#ifdef CONFIG_SMP + +void xen_tlb_flush_all(void) +{ + struct mmuext_op op; + op.cmd = MMUEXT_TLB_FLUSH_ALL; + BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0); +} + +void xen_tlb_flush_mask(cpumask_t mask) +{ + struct mmuext_op op; + op.cmd = MMUEXT_TLB_FLUSH_MULTI; + op.cpuset = mask.bits[0]; + BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0); +} + +void xen_invlpg_all(unsigned long ptr) +{ + struct mmuext_op op; + op.cmd = MMUEXT_INVLPG_ALL; + op.linear_addr = ptr & PAGE_MASK; + BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0); +} + +void xen_invlpg_mask(cpumask_t mask, unsigned long ptr) +{ + struct mmuext_op op; + op.cmd = MMUEXT_INVLPG_MULTI; + op.cpuset = mask.bits[0]; + op.linear_addr = ptr & PAGE_MASK; + BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0); +} + +#endif /* CONFIG_SMP */ + void xen_pgd_pin(unsigned long ptr) { struct mmuext_op op; @@ -169,7 +206,7 @@ unsigned long allocate_empty_lowmem_region(unsigned long pages) phys_to_machine_mapping[__pa(vstart)>>PAGE_SHIFT] = INVALID_P2M_ENTRY; } - xen_tlb_flush(); + flush_tlb_all(); balloon_put_pages(pfn_array, 1 << order); diff --git a/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c b/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c index f73c2664b8..649f64c402 100644 --- a/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c +++ b/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c @@ -260,8 +260,7 @@ static void balloon_process(void *unused) /* Ensure that ballooned highmem pages don't have cached mappings. */ kmap_flush_unused(); - - xen_tlb_flush(); + flush_tlb_all(); /* No more mappings: invalidate pages in P2M and add to balloon. */ for ( i = 0; i < debt; i++ ) diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h index 714a247de3..5333fde72b 100644 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h @@ -407,7 +407,8 @@ extern void noexec_setup(const char *str); do { \ if (__dirty) { \ if ( likely((__vma)->vm_mm == current->mm) ) { \ - HYPERVISOR_update_va_mapping((__address), (__entry), UVMF_INVLPG_LOCAL); \ + HYPERVISOR_update_va_mapping((__address), (__entry), 0); \ + flush_tlb_page((__vma), (__address)); \ } else { \ xen_l1_entry_update((__ptep), (__entry).pte_low); \ flush_tlb_page((__vma), (__address)); \ diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h index 28fcf5a845..4d13a650a2 100644 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h @@ -6,23 +6,11 @@ #include #define __flush_tlb() xen_tlb_flush() - -/* - * Global pages have to be flushed a bit differently. Not a real - * performance problem because this does not happen often. - */ #define __flush_tlb_global() xen_tlb_flush() +#define __flush_tlb_all() xen_tlb_flush() extern unsigned long pgkern_mask; -# define __flush_tlb_all() \ - do { \ - if (cpu_has_pge) \ - __flush_tlb_global(); \ - else \ - __flush_tlb(); \ - } while (0) - #define cpu_has_invlpg (boot_cpu_data.x86 > 3) #define __flush_tlb_single(addr) xen_invlpg(addr) diff --git a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h index 9bc3e4997b..b50c5fc3ad 100644 --- a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h +++ b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h @@ -83,6 +83,14 @@ void xen_pte_unpin(unsigned long ptr); void xen_set_ldt(unsigned long ptr, unsigned long bytes); void xen_machphys_update(unsigned long mfn, unsigned long pfn); +#ifdef CONFIG_SMP +#include +void xen_tlb_flush_all(void); +void xen_invlpg_all(unsigned long ptr); +void xen_tlb_flush_mask(cpumask_t mask); +void xen_invlpg_mask(cpumask_t mask, unsigned long ptr); +#endif + #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) /* ** XXX SMH: 2.4 doesn't have percpu.h (or support SMP guests) so just -- 2.30.2